{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FinRL_Hyperparameter tuning using Optuna",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JwrAr8vEmP2M"
      },
      "source": [
        "# INTRODUCTION\n",
        "1. In this tutorial, we will be tuning hyperparameters for Stable baselines3 models using Optuna.\n",
        "2. The default model hyperparamters may not be adequate for your custom portfolio or custom state-space. Reinforcement learning algorithms are sensitive to hyperparamters, hence tuning is an important step.\n",
        "3. Hyperparamters are tuned based on an objective, which needs to be maximized or minimized. Here we tuned our hyperparamters to maximize the Sharpe Ratio "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qRF-7Vp5NCjU"
      },
      "source": [
        "#Installing FinRL\n",
        "%%capture\n",
        "!pip install git+https://github.com/AI4Finance-LLC/FinRL-Library.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRJk36AZQGuh"
      },
      "source": [
        "#Installing Optuna\n",
        "%%capture\n",
        "!pip3 install optuna"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PqOKn-VWNGt4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "32e0f056-37ad-4af4-f0de-71bcce6e5878"
      },
      "source": [
        "#Importing the libraries\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "# matplotlib.use('Agg')\n",
        "import datetime\n",
        "import optuna\n",
        "%matplotlib inline\n",
        "from finrl.apps import config\n",
        "from optuna.integration import PyTorchLightningPruningCallback\n",
        "from finrl.finrl_meta.preprocessor.yahoodownloader import YahooDownloader\n",
        "from finrl.finrl_meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
        "from finrl.finrl_meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
        "from finrl.finrl_meta.env_stock_trading.env_stocktrading_np import StockTradingEnv as StockTradingEnv_numpy\n",
        "from finrl.drl_agents.stablebaselines3.models import DRLAgent\n",
        "from finrl.drl_agents.rllib.models import DRLAgent as DRLAgent_rllib\n",
        "from finrl.finrl_meta.data_processor import DataProcessor\n",
        "import joblib\n",
        "from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "import ray\n",
        "from pprint import pprint\n",
        "\n",
        "import sys\n",
        "sys.path.append(\"../FinRL-Library\")\n",
        "\n",
        "import itertools"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyfolio/pos.py:27: UserWarning:\n",
            "\n",
            "Module \"zipline.assets\" not found; multipliers will not be applied to position notionals.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z7_fCHS6NMx9"
      },
      "source": [
        "import os\n",
        "if not os.path.exists(\"./\" + config.DATA_SAVE_DIR):\n",
        "    os.makedirs(\"./\" + config.DATA_SAVE_DIR)\n",
        "if not os.path.exists(\"./\" + config.TRAINED_MODEL_DIR):\n",
        "    os.makedirs(\"./\" + config.TRAINED_MODEL_DIR)\n",
        "if not os.path.exists(\"./\" + config.TENSORBOARD_LOG_DIR):\n",
        "    os.makedirs(\"./\" + config.TENSORBOARD_LOG_DIR)\n",
        "if not os.path.exists(\"./\" + config.RESULTS_DIR):\n",
        "    os.makedirs(\"./\" + config.RESULTS_DIR)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "71P6jMlEpikl"
      },
      "source": [
        "## COLLECTING DATA AND PREPROCESSING"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "frT5V9lLOv9X",
        "outputId": "9d05910d-aa9c-4273-ecd3-33e4098eabbb"
      },
      "source": [
        "#Custom ticker list dataframe download\n",
        "ticker_list = config.DOW_30_TICKER\n",
        "df = YahooDownloader(start_date = '2009-01-01',\n",
        "                     end_date = '2021-10-01',\n",
        "                     ticker_list = ticker_list).fetch_data()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (96270, 8)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cntKg5nWO5qn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "93a5b7cb-fc19-446b-da92-a43f070f6049"
      },
      "source": [
        "#You can add technical indicators and turbulence factor to dataframe\n",
        "#Just set the use_technical_indicator=True, use_vix=True and use_turbulence=True\n",
        "fe = FeatureEngineer(\n",
        "                    use_technical_indicator=True,\n",
        "                    tech_indicator_list = config.TECHNICAL_INDICATORS_LIST,\n",
        "                    use_vix=True,\n",
        "                    use_turbulence=True,\n",
        "                    user_defined_feature = False)\n",
        "\n",
        "processed = fe.preprocess_data(df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Successfully added technical indicators\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (3208, 8)\n",
            "Successfully added vix\n",
            "Successfully added turbulence index\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5diXih4zPE6m"
      },
      "source": [
        "list_ticker = processed[\"tic\"].unique().tolist()\n",
        "list_date = list(pd.date_range(processed['date'].min(),processed['date'].max()).astype(str))\n",
        "combination = list(itertools.product(list_date,list_ticker))\n",
        "\n",
        "processed_full = pd.DataFrame(combination,columns=[\"date\",\"tic\"]).merge(processed,on=[\"date\",\"tic\"],how=\"left\")\n",
        "processed_full = processed_full[processed_full['date'].isin(processed['date'])]\n",
        "processed_full = processed_full.sort_values(['date','tic'])\n",
        "\n",
        "processed_full = processed_full.fillna(0)\n",
        "processed_full.sort_values(['date','tic'],ignore_index=True).head(5)\n",
        "\n",
        "processed_full.to_csv('processed_full.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3RrJiFbSPKE2",
        "outputId": "5ca01b3e-8e7d-4e78-8d3f-289b8f3f8674"
      },
      "source": [
        "train = data_split(processed_full, '2009-01-01','2020-07-01')\n",
        "trade = data_split(processed_full, '2020-05-01','2021-10-01')\n",
        "print(len(train))\n",
        "print(len(trade))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "86790\n",
            "10710\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ub4JTTRcPOel",
        "outputId": "432716f9-82ed-463b-c4ac-a0901fe5a0ac"
      },
      "source": [
        "stock_dimension = len(train.tic.unique())\n",
        "state_space = 1 + 2*stock_dimension + len(config.TECHNICAL_INDICATORS_LIST) * stock_dimension\n",
        "print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Stock Dimension: 30, State Space: 181\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YiF95zXgPTsd"
      },
      "source": [
        "#Defining the environment kwargs\n",
        "\n",
        "env_kwargs = {\n",
        "    \"hmax\": 100, \n",
        "    \"initial_amount\": 1000000, \n",
        "    \"buy_cost_pct\": 0.001,\n",
        "    \"sell_cost_pct\": 0.001,\n",
        "    \"state_space\": state_space, \n",
        "    \"stock_dim\": stock_dimension, \n",
        "    \"tech_indicator_list\": config.TECHNICAL_INDICATORS_LIST, \n",
        "    \"action_space\": stock_dimension, \n",
        "    \"reward_scaling\": 1e-4\n",
        "    \n",
        "}\n",
        "#Instantiate the training gym compatible environment\n",
        "e_train_gym = StockTradingEnv(df = train, **env_kwargs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "892NcZALPWHF",
        "outputId": "88d8958a-a9aa-4fbf-9338-c990f7829a1b"
      },
      "source": [
        "#Instantiate the training environment\n",
        "# Also instantiate our training gent\n",
        "env_train, _ = e_train_gym.get_sb_env()\n",
        "print(type(env_train))\n",
        "agent = DRLAgent(env = env_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'stable_baselines3.common.vec_env.dummy_vec_env.DummyVecEnv'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3EwB1T8opX2o"
      },
      "source": [
        "#Instantiate the trading environment\n",
        "e_trade_gym = StockTradingEnv(df = trade, turbulence_threshold = None, **env_kwargs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zOynTQluppye"
      },
      "source": [
        "## TUNING HYPERPARAMETERS USING OPTUNA\n",
        "1. Go to this [link](https://github.com/DLR-RM/rl-baselines3-zoo/blob/master/utils/hyperparams_opt.py), you will find all possible hyperparamters to tune for all the models.\n",
        "2. For your model, grab those hyperparamters which you want to optimize and then return a dictionary of hyperparamters.\n",
        "3. There is a feature in Optuna called as hyperparamters importance, you can point out those hyperparamters which are important for tuning.\n",
        "4. By default Optuna use [TPESampler](https://www.youtube.com/watch?v=tdwgR1AqQ8Y) for sampling hyperparamters from the search space. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_vojRvAsP2ja"
      },
      "source": [
        "def sample_ddpg_params(trial:optuna.Trial):\n",
        "  # Size of the replay buffer\n",
        "  buffer_size = trial.suggest_categorical(\"buffer_size\", [int(1e4), int(1e5), int(1e6)])\n",
        "  learning_rate = trial.suggest_loguniform(\"learning_rate\", 1e-5, 1)\n",
        "  batch_size = trial.suggest_categorical(\"batch_size\", [32, 64, 128, 256, 512])\n",
        "  \n",
        "  return {\"buffer_size\": buffer_size,\n",
        "          \"learning_rate\":learning_rate,\n",
        "          \"batch_size\":batch_size}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xL7LeLeWrj6H"
      },
      "source": [
        "#Calculate the Sharpe ratio\n",
        "#This is our objective for tuning\n",
        "def calculate_sharpe(df):\n",
        "  df['daily_return'] = df['account_value'].pct_change(1)\n",
        "  if df['daily_return'].std() !=0:\n",
        "    sharpe = (252**0.5)*df['daily_return'].mean()/ \\\n",
        "          df['daily_return'].std()\n",
        "    return sharpe\n",
        "  else:\n",
        "    return 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gCRy_kL648DM"
      },
      "source": [
        "## CALLBACKS\n",
        "1. The callback will terminate if the improvement margin is below certain point\n",
        "2. It will terminate after certain number of trial_number are reached, not before that\n",
        "3. It will hold its patience to reach the threshold"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MTPSF3bpHDUT"
      },
      "source": [
        "class LoggingCallback:\n",
        "    def __init__(self,threshold,trial_number,patience):\n",
        "      '''\n",
        "      threshold:int tolerance for increase in sharpe ratio\n",
        "      trial_number: int Prune after minimum number of trials\n",
        "      patience: int patience for the threshold\n",
        "      '''\n",
        "      self.threshold = threshold\n",
        "      self.trial_number  = trial_number\n",
        "      self.patience = patience\n",
        "      self.cb_list = [] #Trials list for which threshold is reached\n",
        "    def __call__(self,study:optuna.study, frozen_trial:optuna.Trial):\n",
        "      #Setting the best value in the current trial\n",
        "      study.set_user_attr(\"previous_best_value\", study.best_value)\n",
        "      \n",
        "      #Checking if the minimum number of trials have pass\n",
        "      if frozen_trial.number >self.trial_number:\n",
        "          previous_best_value = study.user_attrs.get(\"previous_best_value\",None)\n",
        "          #Checking if the previous and current objective values have the same sign\n",
        "          if previous_best_value * study.best_value >=0:\n",
        "              #Checking for the threshold condition\n",
        "              if abs(previous_best_value-study.best_value) < self.threshold: \n",
        "                  self.cb_list.append(frozen_trial.number)\n",
        "                  #If threshold is achieved for the patience amount of time\n",
        "                  if len(self.cb_list)>self.patience:\n",
        "                      print('The study stops now...')\n",
        "                      print('With number',frozen_trial.number ,'and value ',frozen_trial.value)\n",
        "                      print('The previous and current best values are {} and {} respectively'\n",
        "                              .format(previous_best_value, study.best_value))\n",
        "                      study.stop()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QsEKc-9APaS1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 581
        },
        "outputId": "c48f6f45-eca0-4178-b935-3f2f84ee9cfc"
      },
      "source": [
        "from IPython.display import clear_output\n",
        "import sys   \n",
        "\n",
        "os.makedirs(\"models\",exist_ok=True)\n",
        "\n",
        "def objective(trial:optuna.Trial):\n",
        "  #Trial will suggest a set of hyperparamters from the specified range\n",
        "  hyperparameters = sample_ddpg_params(trial)\n",
        "  model_ddpg = agent.get_model(\"ddpg\",model_kwargs = hyperparameters )\n",
        "  #You can increase it for better comparison\n",
        "  trained_ddpg = agent.train_model(model=model_ddpg,\n",
        "                                  tb_log_name=\"ddpg\" ,\n",
        "                             total_timesteps=50000)\n",
        "  trained_ddpg.save('models/ddpg_{}.pth'.format(trial.number))\n",
        "  clear_output(wait=True)\n",
        "  #For the given hyperparamters, determine the account value in the trading period\n",
        "  df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)\n",
        "  #Calculate sharpe from the account value\n",
        "  sharpe = calculate_sharpe(df_account_value)\n",
        "\n",
        "  return sharpe\n",
        "\n",
        "#Create a study object and specify the direction as 'maximize'\n",
        "#As you want to maximize sharpe\n",
        "#Pruner stops not promising iterations\n",
        "#Use a pruner, else you will get error related to divergence of model\n",
        "#You can also use Multivariate samplere\n",
        "#sampler = optuna.samplers.TPESampler(multivarite=True,seed=42)\n",
        "sampler = optuna.samplers.TPESampler(seed=42)\n",
        "study = optuna.create_study(study_name=\"ddpg_study\",direction='maximize',\n",
        "                            sampler = sampler, pruner=optuna.pruners.HyperbandPruner())\n",
        "\n",
        "logging_callback = LoggingCallback(threshold=1e-5,patience=30,trial_number=5)\n",
        "#You can increase the n_trials for a better search space scanning\n",
        "study.optimize(objective, n_trials=30,catch=(ValueError,),callbacks=[logging_callback])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2021-10-16 18:40:58,093]\u001b[0m Trial 12 finished with value: 1.9463535284637115 and parameters: {'buffer_size': 100000, 'learning_rate': 0.0009167785859319845, 'batch_size': 32}. Best is trial 6 with value: 1.9680115561891418.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hit end!\n",
            "{'buffer_size': 100000, 'learning_rate': 0.0004204162637844852, 'batch_size': 32}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_14\n",
            "day: 2892, episode: 250\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3888778.62\n",
            "total_reward: 2888778.62\n",
            "total_cost: 1240.02\n",
            "total_trades: 37682\n",
            "Sharpe: 0.732\n",
            "=================================\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-20-cfc6befe9746>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[0mlogging_callback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLoggingCallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthreshold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpatience\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtrial_number\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;31m#You can increase the n_trials for a better search space scanning\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcatch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mValueError\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlogging_callback\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/optuna/study/study.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m    407\u001b[0m             \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    408\u001b[0m             \u001b[0mgc_after_trial\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgc_after_trial\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 409\u001b[0;31m             \u001b[0mshow_progress_bar\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshow_progress_bar\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    410\u001b[0m         )\n\u001b[1;32m    411\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m     74\u001b[0m                 \u001b[0mreseed_sampler_rng\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m                 \u001b[0mtime_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m                 \u001b[0mprogress_bar\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprogress_bar\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m             )\n\u001b[1;32m     78\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m             \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_run_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m             \u001b[0;32mraise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m         \u001b[0mvalue_or_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    214\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialPruned\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m         \u001b[0;31m# TODO(mamu): Handle multi-objective cases.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-cfc6befe9746>\u001b[0m in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m     11\u001b[0m   trained_ddpg = agent.train_model(model=model_ddpg,\n\u001b[1;32m     12\u001b[0m                                   \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"ddpg\"\u001b[0m \u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m                              total_timesteps=50000)\n\u001b[0m\u001b[1;32m     14\u001b[0m   \u001b[0mtrained_ddpg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'models/ddpg_{}.pth'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m   \u001b[0mclear_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/finrl/drl_agents/stablebaselines3/models.py\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, model, tb_log_name, total_timesteps)\u001b[0m\n\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 135\u001b[0;31m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTensorboardCallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    136\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/stable_baselines3/ddpg/ddpg.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    137\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m             \u001b[0meval_log_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0meval_log_path\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m             \u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m         )\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    211\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m             \u001b[0meval_log_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0meval_log_path\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m             \u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    214\u001b[0m         )\n\u001b[1;32m    215\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    369\u001b[0m                 \u001b[0;31m# Special case when the user passes `gradient_steps=0`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    370\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgradient_steps\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 371\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgradient_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    372\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    373\u001b[0m         \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_training_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, gradient_steps, batch_size)\u001b[0m\n\u001b[1;32m    181\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 183\u001b[0;31m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_target\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    184\u001b[0m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor_target\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    185\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/stable_baselines3/common/utils.py\u001b[0m in \u001b[0;36mpolyak_update\u001b[0;34m(params, target_params, tau)\u001b[0m\n\u001b[1;32m    413\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    414\u001b[0m         \u001b[0;31m# zip does not raise an exception if length of parameters does not match.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 415\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_param\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip_strict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    416\u001b[0m             \u001b[0mtarget_param\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    417\u001b[0m             \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_param\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtarget_param\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/stable_baselines3/common/utils.py\u001b[0m in \u001b[0;36mzip_strict\u001b[0;34m(*iterables)\u001b[0m\n\u001b[1;32m    385\u001b[0m     \u001b[0;31m# Nones in iterable.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m     \u001b[0msentinel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 387\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mcombo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip_longest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0miterables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfillvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msentinel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    388\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0msentinel\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcombo\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Iterables have different lengths\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mparameters\u001b[0;34m(self, recurse)\u001b[0m\n\u001b[1;32m   1442\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1443\u001b[0m         \"\"\"\n\u001b[0;32m-> 1444\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamed_parameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecurse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrecurse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1445\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mnamed_parameters\u001b[0;34m(self, prefix, recurse)\u001b[0m\n\u001b[1;32m   1468\u001b[0m             \u001b[0;32mlambda\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parameters\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1469\u001b[0m             prefix=prefix, recurse=recurse)\n\u001b[0;32m-> 1470\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0melem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgen\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1471\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0melem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1472\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_named_members\u001b[0;34m(self, get_members_fn, prefix, recurse)\u001b[0m\n\u001b[1;32m   1415\u001b[0m             \u001b[0mmembers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_members_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1416\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmembers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1417\u001b[0;31m                 \u001b[0;32mif\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmemo\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1418\u001b[0m                     \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1419\u001b[0m                 \u001b[0mmemo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36m__hash__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    614\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    615\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__hash__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 616\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mhas_torch_function_unary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    617\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mhandle_torch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__hash__\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    618\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sg7LRlHmj9GB",
        "outputId": "b089f7d6-7e92-44f1-b535-5ed127105366"
      },
      "source": [
        "joblib.dump(study, \"final_ddpg_study__.pkl\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['final_ddpg_study__.pkl']"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SVMue1-xuGHC",
        "outputId": "115d9f90-081f-4ef0-e687-2d06eea505fd"
      },
      "source": [
        "#Get the best hyperparamters\n",
        "print('Hyperparameters after tuning',study.best_params)\n",
        "print('Hyperparameters before tuning',config.DDPG_PARAMS)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparameters after tuning {'buffer_size': 100000, 'learning_rate': 9.548041810464153e-05, 'batch_size': 512}\n",
            "Hyperparameters before tuning {'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tsZmMw0ykmYo",
        "outputId": "0da397f4-b412-49f5-aa93-34cdc4bcbf8d"
      },
      "source": [
        "study.best_trial"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "FrozenTrial(number=6, values=[1.9680115561891418], datetime_start=datetime.datetime(2021, 10, 16, 16, 1, 20, 183649), datetime_complete=datetime.datetime(2021, 10, 16, 16, 47, 48, 263917), params={'buffer_size': 100000, 'learning_rate': 9.548041810464153e-05, 'batch_size': 512}, distributions={'buffer_size': CategoricalDistribution(choices=(10000, 100000, 1000000)), 'learning_rate': LogUniformDistribution(high=1.0, low=1e-05), 'batch_size': CategoricalDistribution(choices=(32, 64, 128, 256, 512))}, user_attrs={}, system_attrs={}, intermediate_values={}, trial_id=6, state=TrialState.COMPLETE, value=None)"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fETqKJj4uSi5"
      },
      "source": [
        "from stable_baselines3 import DDPG\n",
        "tuned_model_ddpg = DDPG.load('models/ddpg_{}.pth'.format(study.best_trial.number),env=env_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FgchX1LLuua-",
        "outputId": "00b50778-ae4a-49f8-c676-2d5286d899c0"
      },
      "source": [
        "#Trading period account value with tuned model\n",
        "df_account_value_tuned, df_actions_tuned = DRLAgent.DRL_prediction(\n",
        "    model=tuned_model_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hit end!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8s5KNvVuvr2D",
        "outputId": "6a4e6933-84ca-4689-b8c3-042fc77e181c"
      },
      "source": [
        "#Backtesting with our pruned model\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all_tuned = backtest_stats(account_value=df_account_value_tuned)\n",
        "perf_stats_all_tuned = pd.DataFrame(perf_stats_all_tuned)\n",
        "perf_stats_all_tuned.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_tuned_\"+now+'.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.311337\n",
            "Cumulative returns     0.468121\n",
            "Annual volatility      0.143427\n",
            "Sharpe ratio           1.968012\n",
            "Calmar ratio           3.371128\n",
            "Stability              0.952728\n",
            "Max drawdown          -0.092354\n",
            "Omega ratio            1.404845\n",
            "Sortino ratio          2.816227\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.161312\n",
            "Daily value at risk   -0.016950\n",
            "dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tuGaI9lSvvVD",
        "outputId": "7e72920e-84e5-486f-e7ce-747d282783d2"
      },
      "source": [
        "#Now train with not tuned hyperaparameters\n",
        "#Default config.ddpg_PARAMS\n",
        "non_tuned_model_ddpg = agent.get_model(\"ddpg\",model_kwargs = config.DDPG_PARAMS )\n",
        "trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n",
        "                             tb_log_name='ddpg',\n",
        "                             total_timesteps=50000)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_15\n",
            "---------------------------------\n",
            "| time/              |          |\n",
            "|    episodes        | 4        |\n",
            "|    fps             | 55       |\n",
            "|    time_elapsed    | 209      |\n",
            "|    total timesteps | 11572    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 84       |\n",
            "|    critic_loss     | 1.89e+04 |\n",
            "|    learning_rate   | 0.001    |\n",
            "|    n_updates       | 8679     |\n",
            "|    reward          | 4.206319 |\n",
            "---------------------------------\n",
            "day: 2892, episode: 260\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3610867.99\n",
            "total_reward: 2610867.99\n",
            "total_cost: 999.00\n",
            "total_trades: 54928\n",
            "Sharpe: 0.685\n",
            "=================================\n",
            "---------------------------------\n",
            "| time/              |          |\n",
            "|    episodes        | 8        |\n",
            "|    fps             | 50       |\n",
            "|    time_elapsed    | 455      |\n",
            "|    total timesteps | 23144    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -29.6    |\n",
            "|    critic_loss     | 158      |\n",
            "|    learning_rate   | 0.001    |\n",
            "|    n_updates       | 20251    |\n",
            "|    reward          | 4.206319 |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| time/              |          |\n",
            "|    episodes        | 12       |\n",
            "|    fps             | 49       |\n",
            "|    time_elapsed    | 702      |\n",
            "|    total timesteps | 34716    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -23.7    |\n",
            "|    critic_loss     | 37.4     |\n",
            "|    learning_rate   | 0.001    |\n",
            "|    n_updates       | 31823    |\n",
            "|    reward          | 4.206319 |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| time/              |          |\n",
            "|    episodes        | 16       |\n",
            "|    fps             | 48       |\n",
            "|    time_elapsed    | 946      |\n",
            "|    total timesteps | 46288    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -18      |\n",
            "|    critic_loss     | 16.9     |\n",
            "|    learning_rate   | 0.001    |\n",
            "|    n_updates       | 43395    |\n",
            "|    reward          | 4.206319 |\n",
            "---------------------------------\n",
            "day: 2892, episode: 270\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3610867.99\n",
            "total_reward: 2610867.99\n",
            "total_cost: 999.00\n",
            "total_trades: 54928\n",
            "Sharpe: 0.685\n",
            "=================================\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yZbEYRQ1wBeC",
        "outputId": "c2aacb1f-9fc1-4816-c15a-da512df2b2df"
      },
      "source": [
        "df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hit end!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pFdB4YM3wh0m",
        "outputId": "e31ca771-bf0f-4f5c-d385-839dfa215450"
      },
      "source": [
        "#Backtesting for not tuned hyperparamters\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all = backtest_stats(account_value=df_account_value)\n",
        "perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "# perf_stats_all.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_\"+now+'.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.325768\n",
            "Cumulative returns     0.491062\n",
            "Annual volatility      0.169124\n",
            "Sharpe ratio           1.757594\n",
            "Calmar ratio           3.238950\n",
            "Stability              0.944550\n",
            "Max drawdown          -0.100578\n",
            "Omega ratio            1.355538\n",
            "Sortino ratio          2.590766\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.171057\n",
            "Daily value at risk   -0.020128\n",
            "dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "12fxdvUZwi_W",
        "outputId": "56a25b27-dd6c-4620-acf1-76e68f252f32"
      },
      "source": [
        "#You can see with trial, our sharpe ratio is increasing\n",
        "#Certainly you can afford more number of trials for further optimization\n",
        "from optuna.visualization import plot_optimization_history\n",
        "plot_optimization_history(study)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"08ae1a48-3f9f-46a6-9cf3-3756c1dd2622\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"08ae1a48-3f9f-46a6-9cf3-3756c1dd2622\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '08ae1a48-3f9f-46a6-9cf3-3756c1dd2622',\n",
              "                        [{\"mode\": \"markers\", \"name\": \"Objective Value\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], \"y\": [1.9336748564881099, 1.7103825303995113, 1.451824873143654, 1.399998769842667, 1.610091338541197, 1.9282760290455876, 1.9680115561891418, 1.7307168460129427, 1.8178755898363874, 1.4459502841509786, 1.6852943468510908, 1.8105525374061526, 1.9463535284637115]}, {\"name\": \"Best Value\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], \"y\": [1.9336748564881099, 1.9336748564881099, 1.9336748564881099, 1.9336748564881099, 1.9336748564881099, 1.9336748564881099, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418]}],\n",
              "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Optimization History Plot\"}, \"xaxis\": {\"title\": {\"text\": \"#Trials\"}}, \"yaxis\": {\"title\": {\"text\": \"Objective Value\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('08ae1a48-3f9f-46a6-9cf3-3756c1dd2622');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_TUF2GvAx6-k"
      },
      "source": [
        "from optuna.visualization import plot_contour\n",
        "from optuna.visualization import plot_edf\n",
        "from optuna.visualization import plot_intermediate_values\n",
        "from optuna.visualization import plot_optimization_history\n",
        "from optuna.visualization import plot_parallel_coordinate\n",
        "from optuna.visualization import plot_param_importances\n",
        "from optuna.visualization import plot_slice"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2jkqeSUIyCT0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "outputId": "531590bb-e10c-4a96-b1d7-af57ff166030"
      },
      "source": [
        "#Hyperparamters importance\n",
        "#Ent_coef is the most important\n",
        "plot_param_importances(study)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"b368844c-12b4-48a3-8214-3653f299217d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"b368844c-12b4-48a3-8214-3653f299217d\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        'b368844c-12b4-48a3-8214-3653f299217d',\n",
              "                        [{\"cliponaxis\": false, \"hovertemplate\": [\"learning_rate (LogUniformDistribution): 0.039013664889828874<extra></extra>\", \"buffer_size (CategoricalDistribution): 0.18575466361275744<extra></extra>\", \"batch_size (CategoricalDistribution): 0.7752316714974136<extra></extra>\"], \"marker\": {\"color\": \"rgb(66,146,198)\"}, \"orientation\": \"h\", \"text\": [\"0.039013664889828874\", \"0.18575466361275744\", \"0.7752316714974136\"], \"textposition\": \"outside\", \"texttemplate\": \"%{text:.2f}\", \"type\": \"bar\", \"x\": [0.039013664889828874, 0.18575466361275744, 0.7752316714974136], \"y\": [\"learning_rate\", \"buffer_size\", \"batch_size\"]}],\n",
              "                        {\"showlegend\": false, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Hyperparameter Importances\"}, \"xaxis\": {\"title\": {\"text\": \"Importance for Objective Value\"}}, \"yaxis\": {\"title\": {\"text\": \"Hyperparameter\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('b368844c-12b4-48a3-8214-3653f299217d');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fAD0MIAWukB9"
      },
      "source": [
        "## FURTHER WORKS\n",
        "\n",
        "1.   You can tune more critical hyperparameters\n",
        "2.   Multi-objective hyperparameter optimization using Optuna. Here we can maximize Sharpe and simultaneously minimize Volatility in our account value to tune our hyperparameters\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_edBJqB8yEDr",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "outputId": "2bf34851-7038-4341-b3cb-de7a85608c78"
      },
      "source": [
        "plot_edf(study)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"21d54349-a5ac-4f06-adb0-e17b25a08512\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"21d54349-a5ac-4f06-adb0-e17b25a08512\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '21d54349-a5ac-4f06-adb0-e17b25a08512',\n",
              "                        [{\"mode\": \"lines\", \"name\": \"ddpg_study\", \"type\": \"scatter\", \"x\": [1.399998769842667, 1.4057362727350557, 1.4114737756274442, 1.417211278519833, 1.4229487814122215, 1.4286862843046102, 1.434423787196999, 1.4401612900893874, 1.4458987929817761, 1.4516362958741647, 1.4573737987665534, 1.463111301658942, 1.4688488045513306, 1.4745863074437193, 1.4803238103361078, 1.4860613132284965, 1.4917988161208853, 1.4975363190132738, 1.5032738219056625, 1.509011324798051, 1.5147488276904397, 1.5204863305828284, 1.526223833475217, 1.5319613363676057, 1.5376988392599942, 1.543436342152383, 1.5491738450447716, 1.5549113479371601, 1.5606488508295489, 1.5663863537219374, 1.572123856614326, 1.5778613595067146, 1.5835988623991033, 1.589336365291492, 1.5950738681838805, 1.6008113710762693, 1.606548873968658, 1.6122863768610465, 1.6180238797534352, 1.6237613826458237, 1.6294988855382124, 1.635236388430601, 1.6409738913229897, 1.6467113942153784, 1.6524488971077669, 1.6581864000001556, 1.6639239028925443, 1.6696614057849328, 1.6753989086773216, 1.68113641156971, 1.6868739144620988, 1.6926114173544873, 1.698348920246876, 1.7040864231392647, 1.7098239260316532, 1.715561428924042, 1.7212989318164307, 1.7270364347088192, 1.732773937601208, 1.7385114404935964, 1.7442489433859851, 1.7499864462783736, 1.7557239491707624, 1.761461452063151, 1.7671989549555396, 1.7729364578479283, 1.778673960740317, 1.7844114636327055, 1.7901489665250943, 1.7958864694174828, 1.8016239723098715, 1.80736147520226, 1.8130989780946487, 1.8188364809870374, 1.824573983879426, 1.8303114867718147, 1.8360489896642034, 1.8417864925565919, 1.8475239954489804, 1.853261498341369, 1.8589990012337578, 1.8647365041261463, 1.870474007018535, 1.8762115099109238, 1.8819490128033123, 1.887686515695701, 1.8934240185880897, 1.8991615214804782, 1.9048990243728667, 1.9106365272652555, 1.9163740301576442, 1.9221115330500327, 1.9278490359424214, 1.9335865388348101, 1.9393240417271986, 1.9450615446195874, 1.950799047511976, 1.9565365504043646, 1.962274053296753, 1.9680115561891418], \"y\": [0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.15384615384615385, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.38461538461538464, 0.38461538461538464, 0.38461538461538464, 0.38461538461538464, 0.38461538461538464, 0.46153846153846156, 0.46153846153846156, 0.46153846153846156, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.6153846153846154, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.7692307692307693, 0.8461538461538461, 0.8461538461538461, 0.9230769230769231, 0.9230769230769231, 0.9230769230769231, 1.0]}],\n",
              "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Empirical Distribution Function Plot\"}, \"xaxis\": {\"title\": {\"text\": \"Objective Value\"}}, \"yaxis\": {\"range\": [0, 1], \"title\": {\"text\": \"Cumulative Probability\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('21d54349-a5ac-4f06-adb0-e17b25a08512');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pXTgG1yvgeyq"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}
